#!/usr/bin/env python3
# coding: utf-8

import urllib.request
import sys
import re
from time import sleep

queue = []
pages = set()

def protocol_prefix(url):
    match = re.findall('^(\w+://).*$', url)
    if len(match) > 1:
        print(match)
    if len(match) > 0:
        return match[0]
    else:
        return 'http://'

def print_result():
    res = list(pages)
    res.sort(key=lambda x: x[1])
    for x in res:
        print(x)

def parse_html_references(response, current_url):
    a_tags = re.findall('<[Aa](\s+[^>]*\s+|\s+)href=[\'"]*([^\'"]+)[\'"]*[^>]*>', response)
    urllist = [x[1] for x in a_tags]
    urllist.sort()
    i = 1
    while i < len(urllist):
        if urllist[i - 1] == urllist[i]:
            del urllist[i]
        else:
            i += 1
    i = 0
    while i < len(urllist):
        if urllist[i].startswith('//'):
            urllist[i] = protocol_prefix(current_url) + urllist[i][2:]
        i += 1
    return urllist

def run(url='http://4chan.org/'):
    level = 0
    queue.append((url, level))
    while True:
        url, level = queue.pop(0)
        print('processing {0} (st: {1}) ...'.format(url, level))
        pages.add((url, level))
        print('added to the set ({0} entries)'.format(len(pages)))
        try:
            sleep(0.08)
            response = urllib.request.urlopen(url).read().decode('utf-8')
        except UnicodeDecodeError as e:
            print('failed to parse {0}: unsupported encoding'.format(url))
        except KeyboardInterrupt:
            print('\nABORTING')
            print_result()
            sys.exit()
        except:
            print('unknown error')
        references = parse_html_references(response, url)
        for x in references:
            if x.startswith('http') and x not in pages:
                #print('queueing {0}'.format(x))
                queue.append((x, level + 1))
        print('{0} entries in queue'.format(len(queue)))

if __name__ == "__main__":
    #run('http://mail.ru/')
    run()
    #print(urllib.request.urlopen('http://4chan.org/').read().decode())
